{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Yahoo Webscope S5 Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import shutil\n",
    "from typing import Final\n",
    "from pathlib import Path\n",
    "from collections.abc import Callable\n",
    "from config import data_raw_folder, data_processed_folder\n",
    "from timeeval import Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking for source datasets in /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data and\n",
      "saving processed datasets in /home/projects/akita/data/benchmark-data/data-processed\n"
     ]
    }
   ],
   "source": [
    "dataset_collection_name = \"WebscopeS5\"\n",
    "source_folder = Path(data_raw_folder) / \"Webscope-S5/data\"\n",
    "target_folder = Path(data_processed_folder)\n",
    "\n",
    "print(f\"Looking for source datasets in {source_folder.absolute()} and\\nsaving processed datasets in {target_folder.absolute()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "file handling and transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_regular_folders(path: str) -> list[str]:\n",
    "    files = [os.path.join(path, f) for f in os.listdir(path)]\n",
    "    return [f for f in files if os.path.isdir(f) and not os.path.basename(f).startswith(\".\")]\n",
    "\n",
    "def calc_size(filename: str) -> int:\n",
    "    with open(filename, 'r') as f:\n",
    "        next(f) # skips header\n",
    "        c = 0\n",
    "        for line in f:\n",
    "            c += 1\n",
    "    return c\n",
    "\n",
    "def transform_datetime_column(source: str, target: str, column: str = \"timestamp\") -> None:\n",
    "    df = pd.read_csv(source)\n",
    "    df[\"timestamp\"] = pd.to_datetime(df['timestamp'], unit='h')\n",
    "    df.to_csv(target, index=False)\n",
    "\n",
    "def transform_copy(source: str, target: str) -> None:\n",
    "    try:\n",
    "        shutil.copy(source, target)\n",
    "    except e:\n",
    "        print(f\"Unable to copy file. {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Directories /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5 already exist\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'A2Benchmark': '/home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark',\n",
       " 'A3Benchmark': '/home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark',\n",
       " 'A4Benchmark': '/home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark',\n",
       " 'A1Benchmark': '/home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# shared by all datasets\n",
    "train_is_normal = False\n",
    "input_type = \"univariate\"\n",
    "datetime_index = True\n",
    "\n",
    "# create target directory\n",
    "dataset_subfolder = os.path.join(input_type, dataset_collection_name)\n",
    "target_subfolder = os.path.join(target_folder, dataset_subfolder)\n",
    "try:\n",
    "    os.makedirs(target_subfolder)\n",
    "    print(f\"Created directories {target_subfolder}\")\n",
    "except FileExistsError:\n",
    "    print(f\"Directories {target_subfolder} already exist\")\n",
    "    pass\n",
    "\n",
    "dm = Datasets(target_folder)\n",
    "benchmark_paths = {}\n",
    "for benchmark in list_regular_folders(source_folder):\n",
    "    benchmark_paths[os.path.basename(benchmark)] = benchmark\n",
    "benchmark_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A1Benchmark and A2Benchmark datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_6.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-6.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_3.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-3.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_40.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-40.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_20.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-20.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_4.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-4.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_60.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-60.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_50.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-50.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_31.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-31.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_37.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-37.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_35.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-35.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_26.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-26.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_58.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-58.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_61.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-61.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_29.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-29.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_44.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-44.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_53.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-53.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_41.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-41.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_55.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-55.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_5.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-5.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_51.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-51.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_14.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-14.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_27.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-27.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_30.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-30.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_47.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-47.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_7.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-7.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_63.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-63.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_49.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-49.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_28.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-28.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_45.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-45.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_57.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-57.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_24.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-24.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_15.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-15.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_52.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-52.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_17.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-17.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_46.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-46.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_64.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-64.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_34.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-34.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_2.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-2.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_62.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-62.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_11.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-11.test.csv\n",
      "Skipped source /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/.json\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_8.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-8.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_23.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-23.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_67.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-67.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_32.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-32.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_43.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-43.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_59.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-59.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_19.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-19.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_56.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-56.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_10.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-10.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_36.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-36.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_12.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-12.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_54.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-54.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_25.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-25.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_9.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-9.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_65.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-65.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_33.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-33.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_48.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-48.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_18.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-18.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_39.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-39.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_42.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-42.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_16.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-16.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_66.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-66.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_13.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-13.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_38.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-38.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_21.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-21.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_1.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-1.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A1Benchmark/real_22.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A1Benchmark-22.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_17.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-17.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_65.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-65.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_53.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-53.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_66.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-66.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_5.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-5.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_52.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-52.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_26.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-26.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_91.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-91.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_19.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-19.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_73.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-73.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_47.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-47.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_89.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-89.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_16.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-16.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_97.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-97.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_30.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-30.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_70.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-70.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_98.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-98.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_93.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-93.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_69.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-69.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_24.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-24.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_77.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-77.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_85.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-85.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_51.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-51.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_81.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-81.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_7.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-7.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_28.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-28.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_99.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-99.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_38.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-38.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_3.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-3.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_83.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-83.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_88.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-88.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_21.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-21.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_76.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-76.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_8.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-8.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_59.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-59.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_9.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-9.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_60.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-60.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_12.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-12.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_10.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-10.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_1.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-1.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_36.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-36.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_90.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-90.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_64.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-64.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_61.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-61.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_58.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-58.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_45.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-45.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_22.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-22.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_41.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-41.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_56.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-56.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_86.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-86.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_78.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-78.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_68.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-68.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_20.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-20.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_6.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-6.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_55.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-55.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_79.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-79.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_94.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-94.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_100.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-100.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_18.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-18.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_37.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-37.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_80.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-80.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_44.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-44.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_95.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-95.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_46.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-46.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_87.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-87.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_35.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-35.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_32.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-32.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_39.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-39.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_40.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-40.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_63.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-63.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_34.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-34.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_11.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-11.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_72.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-72.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_75.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-75.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_92.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-92.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_49.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-49.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_62.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-62.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_67.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-67.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_74.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-74.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_31.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-31.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_27.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-27.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_25.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-25.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_82.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-82.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_54.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-54.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_84.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-84.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_42.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-42.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_50.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-50.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_15.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-15.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_33.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-33.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_23.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-23.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_29.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-29.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_71.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-71.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_96.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-96.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_14.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-14.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_4.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-4.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_43.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-43.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_57.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-57.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_13.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-13.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_48.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-48.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A2Benchmark/synthetic_2.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A2Benchmark-2.test.csv\n"
     ]
    }
   ],
   "source": [
    "# shared by the datasets from A1 and A2\n",
    "train_type = \"unsupervised\"\n",
    "\n",
    "for benchmark, dataset_type in [(\"A1Benchmark\", \"real\"), (\"A2Benchmark\", \"synthetic\")]:\n",
    "    # dataset transformation\n",
    "    transform_file: Callable[[str, str], None] = transform_datetime_column if benchmark == \"A1Benchmark\" else transform_copy\n",
    "\n",
    "    for f in os.listdir(benchmark_paths[benchmark]):\n",
    "        source_file = os.path.join(benchmark_paths[benchmark], f)\n",
    "        if (os.path.isfile(source_file) and not f == \".json\"):\n",
    "            # get file number for target filename\n",
    "            dataset_number = os.path.splitext(os.path.basename(f))[0].split(\"_\")[1]\n",
    "            filename = f\"{benchmark}-{dataset_number}.test.csv\"\n",
    "\n",
    "            # save metadata\n",
    "            dataset_name = filename.split(\".\")[0]\n",
    "            path = os.path.join(dataset_subfolder, filename)\n",
    "            target_filepath = os.path.join(target_subfolder, filename)\n",
    "            dataset_length = calc_size(source_file)\n",
    "            dm.add_dataset((dataset_collection_name, dataset_name),\n",
    "                train_path = None,\n",
    "                test_path = path,\n",
    "                dataset_type = dataset_type,\n",
    "                datetime_index = datetime_index,\n",
    "                split_at = None,\n",
    "                train_type = train_type,\n",
    "                train_is_normal = train_is_normal,\n",
    "                input_type = input_type,\n",
    "                dataset_length = dataset_length\n",
    "            )\n",
    "            # transform file\n",
    "            transform_file(source_file, target_filepath)\n",
    "            print(f\"Processed source dataset {source_file} -> {target_filepath}\")\n",
    "        else:\n",
    "            print(f\"Skipped source {source_file}\")\n",
    "\n",
    "# save metadata of benchmark\n",
    "dm.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A3Benchmark and A4Benchmark datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS76.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-76.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS67.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-67.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS97.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-97.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS98.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-98.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS24.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-24.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS52.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-52.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS4.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-4.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS10.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-10.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS20.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-20.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS82.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-82.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS70.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-70.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS14.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-14.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS12.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-12.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS55.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-55.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS2.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-2.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS57.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-57.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS3.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-3.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS87.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-87.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS99.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-99.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS65.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-65.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS95.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-95.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS77.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-77.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS31.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-31.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS23.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-23.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS54.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-54.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS88.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-88.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS83.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-83.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS94.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-94.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS8.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-8.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS91.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-91.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS36.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-36.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS79.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-79.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS25.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-25.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS41.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-41.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS60.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-60.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS1.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-1.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS44.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-44.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS42.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-42.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS39.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-39.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS63.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-63.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS86.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-86.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS49.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-49.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS33.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-33.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS29.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-29.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS28.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-28.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS73.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-73.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS37.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-37.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS75.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-75.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS15.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-15.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS69.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-69.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS72.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-72.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS27.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-27.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS5.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-5.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS71.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-71.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS92.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-92.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS47.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-47.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS32.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-32.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS40.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-40.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS85.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-85.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS19.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-19.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS38.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-38.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS51.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-51.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS78.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-78.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS90.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-90.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS17.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-17.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS58.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-58.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS9.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-9.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS35.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-35.test.csv\n",
      "Skipped source /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark_all.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS30.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-30.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS13.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-13.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS6.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-6.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS53.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-53.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS84.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-84.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS61.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-61.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS56.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-56.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS93.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-93.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS100.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-100.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS68.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-68.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS50.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-50.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS89.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-89.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS62.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-62.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS64.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-64.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS16.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-16.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS46.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-46.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS81.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-81.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS48.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-48.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS22.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-22.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS18.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-18.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS96.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-96.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS34.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-34.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS7.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-7.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS80.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-80.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS66.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-66.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS59.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-59.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS45.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-45.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS43.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-43.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS74.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-74.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS21.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-21.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS11.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-11.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A3Benchmark/A3Benchmark-TS26.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A3Benchmark-26.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS8.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-8.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS38.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-38.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS62.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-62.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS96.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-96.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS59.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-59.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS40.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-40.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS75.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-75.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS25.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-25.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS37.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-37.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS50.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-50.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS43.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-43.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS55.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-55.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS58.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-58.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS26.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-26.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS28.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-28.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS90.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-90.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS22.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-22.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS52.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-52.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS66.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-66.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS4.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-4.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS76.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-76.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS41.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-41.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS97.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-97.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS95.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-95.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS46.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-46.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS18.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-18.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS91.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-91.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS16.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-16.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS42.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-42.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS32.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-32.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS21.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-21.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS77.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-77.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS78.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-78.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS60.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-60.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS35.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-35.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS29.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-29.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS11.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-11.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS86.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-86.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS85.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-85.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS13.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-13.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS53.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-53.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS54.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-54.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS80.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-80.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS69.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-69.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS2.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-2.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS1.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-1.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS31.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-31.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS45.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-45.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS83.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-83.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS10.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-10.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS67.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-67.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS7.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-7.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS100.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-100.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS57.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-57.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS17.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-17.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS74.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-74.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS5.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-5.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS12.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-12.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS48.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-48.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS82.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-82.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS89.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-89.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS15.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-15.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS71.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-71.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS70.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-70.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS6.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-6.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS79.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-79.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS19.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-19.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS64.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-64.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS87.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-87.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS51.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-51.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS99.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-99.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS24.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-24.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS20.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-20.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS39.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-39.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS27.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-27.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS93.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-93.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS84.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-84.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS34.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-34.test.csv\n",
      "Skipped source /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark_all.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS36.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-36.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS49.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-49.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS56.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-56.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS73.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-73.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS81.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-81.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS23.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-23.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS72.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-72.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS44.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-44.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS92.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-92.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS63.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-63.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS9.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-9.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS3.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-3.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS14.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-14.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS30.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-30.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS65.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-65.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS68.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-68.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS98.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-98.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS94.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-94.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS47.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-47.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS61.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-61.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS88.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-88.test.csv\n",
      "Processed source dataset /home/projects/akita/data/benchmark-data/data-raw/Webscope-S5/data/A4Benchmark/A4Benchmark-TS33.csv -> /home/projects/akita/data/benchmark-data/data-processed/univariate/WebscopeS5/A4Benchmark-33.test.csv\n"
     ]
    }
   ],
   "source": [
    "# shared by the datasets from A3 and A4\n",
    "dataset_type = \"synthetic\"\n",
    "train_type = \"unsupervised\"\n",
    "train_is_normal = False\n",
    "input_type = \"univariate\"\n",
    "datetime_index = True\n",
    "\n",
    "# dataset transformation\n",
    "def transform_clean(source: str, target: str) -> None:\n",
    "    df = pd.read_csv(source)\n",
    "    # parse as unix timestamp in seconds\n",
    "    df[\"timestamp\"] = pd.to_datetime(df[\"timestamps\"], unit='s')\n",
    "    # both anomalies and changepoints are anomalous\n",
    "    df[\"is_anomaly\"] = df[\"anomaly\"] | df[\"changepoint\"]\n",
    "    # remove information about seasonality and trend\n",
    "    df = df[[\"timestamp\", \"value\", \"is_anomaly\"]]\n",
    "    df.to_csv(target, index=False)\n",
    "\n",
    "for benchmark in [\"A3Benchmark\", \"A4Benchmark\"]:\n",
    "    # dataset transformation\n",
    "    transform_file: Callable[[str, str], None] = transform_clean\n",
    "\n",
    "    for f in os.listdir(benchmark_paths[benchmark]):\n",
    "        source_file = os.path.join(benchmark_paths[benchmark], f)\n",
    "        if (os.path.isfile(source_file) and not f.endswith(\"all.csv\")):\n",
    "            # get file number for target filename\n",
    "            dataset_number = os.path.splitext(os.path.basename(f))[0].split(\"-\")[1][2:]\n",
    "            filename = f\"{benchmark}-{dataset_number}.test.csv\"\n",
    "\n",
    "            # save metadata\n",
    "            dataset_name = filename.split(\".\")[0]\n",
    "            path = os.path.join(dataset_subfolder, filename)\n",
    "            target_filepath = os.path.join(target_subfolder, filename)\n",
    "            dataset_length = calc_size(source_file)\n",
    "            dm.add_dataset((dataset_collection_name, dataset_name),\n",
    "                train_path = None,\n",
    "                test_path = path,\n",
    "                dataset_type = dataset_type,\n",
    "                datetime_index = datetime_index,\n",
    "                split_at = None,\n",
    "                train_type = train_type,\n",
    "                train_is_normal = train_is_normal,\n",
    "                input_type = input_type,\n",
    "                dataset_length = dataset_length\n",
    "            )\n",
    "            # transform file\n",
    "            transform_file(source_file, target_filepath)\n",
    "            print(f\"Processed source dataset {source_file} -> {target_filepath}\")\n",
    "        else:\n",
    "            print(f\"Skipped source {source_file}\")\n",
    "\n",
    "# save metadata of benchmark\n",
    "dm.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>train_path</th>\n",
       "      <th>test_path</th>\n",
       "      <th>dataset_type</th>\n",
       "      <th>datetime_index</th>\n",
       "      <th>split_at</th>\n",
       "      <th>train_type</th>\n",
       "      <th>train_is_normal</th>\n",
       "      <th>input_type</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>collection_name</th>\n",
       "      <th>dataset_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"11\" valign=\"top\">WebscopeS5</th>\n",
       "      <th>A1Benchmark-1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A1Benchmark-1.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A1Benchmark-10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A1Benchmark-10.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A1Benchmark-11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A1Benchmark-11.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A1Benchmark-12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A1Benchmark-12.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A1Benchmark-13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A1Benchmark-13.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A4Benchmark-95</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A4Benchmark-95.test.csv</td>\n",
       "      <td>synthetic</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A4Benchmark-96</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A4Benchmark-96.test.csv</td>\n",
       "      <td>synthetic</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A4Benchmark-97</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A4Benchmark-97.test.csv</td>\n",
       "      <td>synthetic</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A4Benchmark-98</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A4Benchmark-98.test.csv</td>\n",
       "      <td>synthetic</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A4Benchmark-99</th>\n",
       "      <td>NaN</td>\n",
       "      <td>univariate/WebscopeS5/A4Benchmark-99.test.csv</td>\n",
       "      <td>synthetic</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>univariate</td>\n",
       "      <td>1680</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>367 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                               train_path  \\\n",
       "collection_name dataset_name                \n",
       "WebscopeS5      A1Benchmark-1         NaN   \n",
       "                A1Benchmark-10        NaN   \n",
       "                A1Benchmark-11        NaN   \n",
       "                A1Benchmark-12        NaN   \n",
       "                A1Benchmark-13        NaN   \n",
       "...                                   ...   \n",
       "                A4Benchmark-95        NaN   \n",
       "                A4Benchmark-96        NaN   \n",
       "                A4Benchmark-97        NaN   \n",
       "                A4Benchmark-98        NaN   \n",
       "                A4Benchmark-99        NaN   \n",
       "\n",
       "                                                                    test_path  \\\n",
       "collection_name dataset_name                                                    \n",
       "WebscopeS5      A1Benchmark-1    univariate/WebscopeS5/A1Benchmark-1.test.csv   \n",
       "                A1Benchmark-10  univariate/WebscopeS5/A1Benchmark-10.test.csv   \n",
       "                A1Benchmark-11  univariate/WebscopeS5/A1Benchmark-11.test.csv   \n",
       "                A1Benchmark-12  univariate/WebscopeS5/A1Benchmark-12.test.csv   \n",
       "                A1Benchmark-13  univariate/WebscopeS5/A1Benchmark-13.test.csv   \n",
       "...                                                                       ...   \n",
       "                A4Benchmark-95  univariate/WebscopeS5/A4Benchmark-95.test.csv   \n",
       "                A4Benchmark-96  univariate/WebscopeS5/A4Benchmark-96.test.csv   \n",
       "                A4Benchmark-97  univariate/WebscopeS5/A4Benchmark-97.test.csv   \n",
       "                A4Benchmark-98  univariate/WebscopeS5/A4Benchmark-98.test.csv   \n",
       "                A4Benchmark-99  univariate/WebscopeS5/A4Benchmark-99.test.csv   \n",
       "\n",
       "                               dataset_type  datetime_index  split_at  \\\n",
       "collection_name dataset_name                                            \n",
       "WebscopeS5      A1Benchmark-1          real            True       NaN   \n",
       "                A1Benchmark-10         real            True       NaN   \n",
       "                A1Benchmark-11         real            True       NaN   \n",
       "                A1Benchmark-12         real            True       NaN   \n",
       "                A1Benchmark-13         real            True       NaN   \n",
       "...                                     ...             ...       ...   \n",
       "                A4Benchmark-95    synthetic            True       NaN   \n",
       "                A4Benchmark-96    synthetic            True       NaN   \n",
       "                A4Benchmark-97    synthetic            True       NaN   \n",
       "                A4Benchmark-98    synthetic            True       NaN   \n",
       "                A4Benchmark-99    synthetic            True       NaN   \n",
       "\n",
       "                                  train_type  train_is_normal  input_type  \\\n",
       "collection_name dataset_name                                                \n",
       "WebscopeS5      A1Benchmark-1   unsupervised            False  univariate   \n",
       "                A1Benchmark-10  unsupervised            False  univariate   \n",
       "                A1Benchmark-11  unsupervised            False  univariate   \n",
       "                A1Benchmark-12  unsupervised            False  univariate   \n",
       "                A1Benchmark-13  unsupervised            False  univariate   \n",
       "...                                      ...              ...         ...   \n",
       "                A4Benchmark-95  unsupervised            False  univariate   \n",
       "                A4Benchmark-96  unsupervised            False  univariate   \n",
       "                A4Benchmark-97  unsupervised            False  univariate   \n",
       "                A4Benchmark-98  unsupervised            False  univariate   \n",
       "                A4Benchmark-99  unsupervised            False  univariate   \n",
       "\n",
       "                                length  \n",
       "collection_name dataset_name            \n",
       "WebscopeS5      A1Benchmark-1     1420  \n",
       "                A1Benchmark-10    1439  \n",
       "                A1Benchmark-11    1439  \n",
       "                A1Benchmark-12    1439  \n",
       "                A1Benchmark-13    1439  \n",
       "...                                ...  \n",
       "                A4Benchmark-95    1680  \n",
       "                A4Benchmark-96    1680  \n",
       "                A4Benchmark-97    1680  \n",
       "                A4Benchmark-98    1680  \n",
       "                A4Benchmark-99    1680  \n",
       "\n",
       "[367 rows x 9 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dm.refresh()\n",
    "dm.df().loc[(slice(dataset_collection_name, dataset_collection_name), slice(None))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Datasets\t733\n",
      "Combined length\t123542562\n"
     ]
    }
   ],
   "source": [
    "print(f\"Datasets\\t{len(dm._df)}\")\n",
    "points = int(dm._df[\"length\"].sum())\n",
    "print(f\"Combined length\\t{points}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with DatasetMetadata(\".\") as dm:\n",
    "    dm.add_datasets(\n",
    "        dataset_name,\n",
    "        dataset_collection_name,\n",
    "        None,\n",
    "        path,\n",
    "        \"real\",\n",
    "        datetime_index,\n",
    "        split_at,\n",
    "        train_type,\n",
    "        train_is_normal,\n",
    "        input_type,\n",
    "        dataset_length\n",
    "    )\n",
    "dm.refresh()\n",
    "data = [{\n",
    "    \"dataset_name\": dataset_name,\n",
    "    \"collection_name\": dataset_collection_name,\n",
    "    \"test_path\": path,\n",
    "    \"dataset_type\": dataset_type,\n",
    "    \"datetime_index\": datetime_index,\n",
    "    \"split_at\": split_at,\n",
    "    \"train_type\": \"unknown\",\n",
    "    \"train_is_normal\": train_is_normal,\n",
    "    \"input_type\": input_type,\n",
    "    \"length\": dataset_length\n",
    "},{\n",
    "    \"dataset_name\": \"A1Benchmark-18\",\n",
    "    \"collection_name\": dataset_collection_name,\n",
    "    \"test_path\": path,\n",
    "    \"dataset_type\": dataset_type,\n",
    "    \"datetime_index\": datetime_index,\n",
    "    \"split_at\": split_at,\n",
    "    \"train_type\": train_type,\n",
    "    \"train_is_normal\": train_is_normal,\n",
    "    \"input_type\": input_type,\n",
    "    \"length\": dataset_length\n",
    "}]\n",
    "dm.add_datasets(data)\n",
    "dm._df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "timeeval",
   "language": "python",
   "name": "timeeval"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
